# -*- coding: utf-8 -*-
'''
Created on Apr 20, 2013

@author: LONG HOANG GIANG
'''
import os, sys
from urlparse import urljoin
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
import logging
import re
import pyLib
import json
import workerpool


logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', datefmt='%d/%m/%Y %H:%M:%S')

output_path = "/longhoanggiang/truyentext/__doctruyen_mobi"
output_path_files = output_path + "/files"
pyLib.createIfNotExistsPath(output_path_files)


def getDetail(item):
    tree = pyLib.loadWeb(item['url']).build_tree()
    contentNode = tree.xpath("//div[@class='postbg']")[0]
    text = pyLib.cleanHTML(pyLib.Etree.tostring(contentNode))
    print text
    html = '''<b>{0}</b><br /><hr /><br />{1}'''.format(item['name'], text)
    html = re.sub(r"\n", "<br />", html)
    pyLib.gzip(output_path_files + "/" + item['path'], json.dumps(html))


def process():
    
    data = []
    for page in range(1, 64):
        url = "http://doctruyen.mobi/truyen-sex-nguoi-lon/index{0}.html".format(page)
        tree = pyLib.loadWeb(url).build_tree()
        for node in tree.xpath("//a[contains(@id, 'thread_gotonew_')]"):
            name = pyLib.stringify(node)
            if name == '»»»': continue
            print name
            link = node.get('href', '')
            if link == '': continue
            link = urljoin(url, link)
            if type(link).__name__ == 'unicode': link = link.encode('utf-8')
            data.append({'name': name, 'url': link, 'path': pyLib.crc32unsigned(link)})
    
    pool = workerpool.WorkerPool(size=5)
    pool.map(getDetail, data)
    pool.shutdown()
    pool.wait()
    
    for item in data:
        del item['url']
        
    pyLib.gzip(output_path + "/data", pyLib.encryptCipher(json.dumps(data)))
    
    
            
if __name__ == '__main__':
    
    process()
    
    logging.info("finished")
    os._exit(1)